#!/anaconda3/envs/FEALPy/bin python3.8
# -*- coding: utf-8 -*-
# File: laptop_model_new.py
# Author: Bryan SHEN
# E-mail: m18801919240_3@163.com
# Site: Shanghai, China
# Time: 2024/2/27 13:51
# File-Desp: 多个经典机器学习模型vote + result线性回归分析的形式

import datetime
import json
import os
import pickle
import time
import math
import numpy as np
import pandas as pd
import requests
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder, StandardScaler, OrdinalEncoder
from tensorflow.python.keras import regularizers

from src.laptop.anomaly import anomaly_file_loader as af_loader
from src.laptop.anomaly.anomaly_detect import LaptopAnomalyDetector
from src.laptop.anomaly.anomaly_load import load_anomaly_document_item
from src.laptop.model import laptop_model_config
from src.laptop.preprocessor.fish_product_data_loader import preprocess_fish_product
from src.laptop.preprocessor.settle_data_loader import preprocess_settle_data
from src.laptop.preprocessor.model_data_preprocessor import preprocess_model_data, random_ratio
from src.utils.config import logger, config
from src.utils.db_processor import mysql_prediction_processor, mysql_price_model, presto_processor
from src.utils.feishu_message import feishu_messager
from src.utils.util import check_conflict_file, get_today, format_date_string
from src.laptop.model.laptop_model_config import ALL_FEATURES
from src.utils.model_utils import getMape, custom_mean_absolute_percentage
from src.laptop.job.sku_price_complement import COMPLETE_DATA_SQL, LEVEL_SQL, LEVEL_TEMPLATE_MAPPING_SQL, BRAND_ID_NAME_SQL, query_sku_sql
from src.utils.redis_pool import redis_conn
from src.laptop.predict import laptop_predict_config
from sklearn.utils import shuffle

from sklearn.preprocessing import LabelEncoder, MinMaxScaler, Normalizer, StandardScaler , OneHotEncoder
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import QuantileTransformer
from sklearn.model_selection import GridSearchCV, train_test_split

from xgboost import XGBRFRegressor
from sklearn.linear_model import LinearRegression, Lasso, Ridge
from sklearn.tree import DecisionTreeRegressor, ExtraTreeRegressor
from sklearn.ensemble import AdaBoostRegressor, GradientBoostingRegressor, RandomForestRegressor

from sklearn.metrics import r2_score, mean_absolute_error, mean_squared_error


class LaptopModelNew(object):
    """
    笔记本价格模型构建过程
        1. 读取结算数据
        2. 排除异常数据
        3. 检测当日出货物品异常数据
        4. 训练模型
        5. 检测价格趋势异常数据
        6. 处理钓鱼物品
        7. 评估模型
        8. 保存模型
        9. 推送模型
        10. 调用清除缓存接口
    """

    def __init__(self, model_date=None):

        # 建模日期
        if model_date is None:
            self.model_date = get_today()
        else:
            self.model_date = model_date
        # 模型训练数据初始化
        self.model_data = None
        self.model = None
        self.train_history = None
        self.mysql_price_num = 0
        # 特征变量One-Hot编码处理器
        self.ohe = OneHotEncoder(handle_unknown='ignore')
        self.standard = StandardScaler()
        self.ode = OrdinalEncoder()
        self.scaler = StandardScaler()  # 价格标准化
        self.push_model_file_list = []  # 模型推送文件列表
        self.ab_price_api_url = config.get_config('ab_price_api', 'cache_clear_url')  # api缓存清除url
        self.history_avg_price = pd.DataFrame({'product_sku_key': [], 'product_level_key': [], 'avg_price': []})

        self.brand_related_features = None  # 品牌相关的特征
        self.product_relate_features = None  # 产品相关的特征
        logger.info('self.model_date is@{}'.format(self.model_date))

    def load_model_data(self):
        """ 加载模型训练数据 """

        # 这里的onehot模型只是为了查看型号是否为新型号
        self.model_data, self.product_ohe = preprocess_settle_data(model_date=self.model_date)
        logger.info('model_data1 shape@{}'.format(self.model_data.shape))

        # 只提取预处理后3个flag为0的数据
        self.model_data = self.model_data[self.model_data['ssd_flag'] == 0]
        self.model_data = self.model_data[self.model_data['memory_flag'] == 0]
        self.model_data = self.model_data[self.model_data['hd_flag'] == 0]
        self.model_data['sku_level'] = self.model_data['secondary_sku_id'].astype(str) + '_' + self.model_data['secondary_level_name'].astype(str)
        logger.info('model_data2 shape@{}'.format(self.model_data.shape))

    def eliminate_anomaly(self):
        """ 清除异常数，主要是以下两类：
            1、刷单数据
            2、数据库异常数据
            3、部分异常字段
        """
        logger.info('eliminating anomaly...')
        # 1、剔除刷单订单和刷单产品数据
        # 刷单数据文件
        scalping_document_id = af_loader.load_scalping_document_id()  #  读取刷单订单号列表
        scalping_product_no = af_loader.load_scalping_product_no()  # 读取刷单产品列表
        # 剔除刷单数据
        self.model_data = self.model_data[~self.model_data['document_item_id'].isin(scalping_document_id)]
        self.model_data = self.model_data[~self.model_data['product_no'].isin(scalping_product_no)]
        # 2、数据库异常数据
        anomaly_document_item = load_anomaly_document_item()
        if not anomaly_document_item.empty:
            logger.info('eliminating anomaly products size@{}'.format(anomaly_document_item.shape))
            self.model_data = self.model_data[~self.model_data['document_item_id'].isin(anomaly_document_item['document_item_id'])]
        # 3、剔除个别异常值
        # 价格小于等于0的数据
        self.model_data = self.model_data[self.model_data['item_quotation_price_num'] > 0]
        self.model_data = self.model_data[self.model_data['product_name'] != '不可开机笔记本']

    def prepare_data(self):
        """ 数据预处理及训练数据准备 """

        is_test = 0
        RECENT_DAYS = 3  # 提取过去3天的数据
        if is_test == 0:
            # 线上部署
            recent_date = self.model_date - datetime.timedelta(days=RECENT_DAYS)
            self.recent_data = self.model_data[(self.model_data['shop_out_date'] > recent_date) & (self.model_data['shop_out_date'] <= self.model_date)]  # kr
            self.train_data = self.model_data
        elif is_test == 1:
            # 测试：训练集train_data使用T-2天及之前数据时
            recent_date = self.model_date
            train_date = self.model_date - datetime.timedelta(days=1)
            self.recent_data = self.model_data[self.model_data['shop_out_date'] == recent_date]
            self.train_data = self.model_data[self.model_data['shop_out_date'] < train_date]
        else:
            # 测试：训练集train_data使用T-1天及之前数据时
            recent_date = self.model_date - datetime.timedelta(days=1)
            train_date = self.model_date - datetime.timedelta(days=1)
            self.recent_data = self.model_data[self.model_data['shop_out_date'] == recent_date]  # 用来看模型拟合能力
            self.train_data = self.model_data[self.model_data['shop_out_date'] <= train_date]

    def additional_features(self):
        """ 生成更品牌brand和产品product相关的价格price特征 """

        # 品牌-价格特征
        train_brand_groupby = self.train_data.groupby("product_brand_id")
        all_info = {}
        for brand_id, brand_data in train_brand_groupby:
            info = {}
            brand_data = brand_data[brand_data['item_quotation_price_num'] > 0]
            info['brand_amount'] = len(brand_data)
            info['brand_price_max'] = brand_data.item_quotation_price_num.max()
            info['brand_price_median'] = brand_data.item_quotation_price_num.median()
            info['brand_price_min'] = brand_data.item_quotation_price_num.min()
            info['brand_price_sum'] = brand_data.item_quotation_price_num.sum()
            info['brand_price_std'] = brand_data.item_quotation_price_num.std()
            info['brand_price_mean'] = brand_data.item_quotation_price_num.mean()
            info['brand_price_skew'] = brand_data.item_quotation_price_num.skew()
            info['brand_price_kurt'] = brand_data.item_quotation_price_num.kurt()
            info['brand_price_mad'] = brand_data.item_quotation_price_num.mad()

            all_info[brand_id] = info

        self.brand_related_features = pd.DataFrame(all_info).T.reset_index().rename(columns={"index": "product_brand_id"})
        self.train_data = self.train_data.merge(self.brand_related_features, how='left', on='product_brand_id')

        # 产品-价格特征
        train_product_groupby = self.train_data.groupby("product_id")
        all_info = {}
        for product_id, product_data in train_product_groupby:
            info = {}
            product_data = product_data[product_data['item_quotation_price_num'] > 0]
            info['product_amount'] = len(product_data)
            all_info[product_id] = info

        self.product_relate_features = pd.DataFrame(all_info).T.reset_index().rename(columns={"index": "product_id"})
        self.train_data = self.train_data.merge(self.product_relate_features, how='left', on='product_id')

        # return data, brand_fe, product_fe

    def get_model_available_data(self, x, y, is_train=False, is_standard=False):
        """ 对数值型和类别型，进行不同的特征处理 """

        # 类别型特征
        if is_train:
            x_ava = self.ohe.fit_transform(x[laptop_model_config.CLASSIFICATION_FEATURES])
        else:
            x_ava = self.ohe.transform(x[laptop_model_config.CLASSIFICATION_FEATURES])

        # 数值型特征
        if is_standard:
            if is_train:
                x_ava_continuous = self.standard.fit_transform(x[laptop_model_config.CONTINUOUS_FEATURES])
            else:
                x_ava_continuous = self.standard.transform(x[laptop_model_config.CONTINUOUS_FEATURES])
        else:
            x_ava_continuous = np.array(x[laptop_model_config.CONTINUOUS_FEATURES])

        y_ava = y
        return x_ava, x_ava_continuous, y_ava

    def build_features(self):
        """ 构建数据特征 """

        # 生成更品牌brand和产品product相关的价格price特征
        self.additional_features()

        self.train_data.fillna(self.train_data.median(), inplace=True)
        self.recent_data = self.recent_data.merge(self.brand_related_features, how='left', on='product_brand_id')
        self.recent_data = self.recent_data.merge(self.product_relate_features, how='left', on='product_id')
        self.recent_data['brand_amount'].fillna(1, inplace=True)
        self.recent_data['brand_price_std'].fillna(0, inplace=True)
        self.recent_data.fillna({'brand_price_max': 200, 'brand_price_mean': 200, 'brand_price_median': 200, 'brand_price_min': 200}, inplace=True)
        self.recent_data['product_amount'].fillna(1, inplace=True)
        self.median_dict = dict(self.train_data.median())
        self.recent_data.fillna(self.median_dict, inplace=True)

        # 设置训练预测集数据类型
        self.train_data[laptop_model_config.CLASSIFICATION_FEATURES] = self.train_data[laptop_model_config.CLASSIFICATION_FEATURES].astype(str)
        self.train_data[laptop_model_config.CONTINUOUS_FEATURES] = self.train_data[laptop_model_config.CONTINUOUS_FEATURES].astype(float)
        self.recent_data[laptop_model_config.CLASSIFICATION_FEATURES] = self.recent_data[laptop_model_config.CLASSIFICATION_FEATURES].astype(str)
        self.recent_data[laptop_model_config.CONTINUOUS_FEATURES] = self.recent_data[laptop_model_config.CONTINUOUS_FEATURES].astype(float)

        # train_data['weight'] = train_data['settle_list_create_date'].apply(self.weight_function)
        self.train_data['weight'] = 1

        # self.train_data = self.train_data[self.train_data['product_name'] != '不可开机笔记本']  # 数据预处理已做过该操作
        self.train_data = shuffle(self.train_data, random_state=1)
        
        split_flag = 1
        is_standard = True
        if split_flag:
            # x_valid 只使用opt的数据
            self.train_data_opt = self.train_data[self.train_data['is_pop']==0]
            self.train_data_pop = self.train_data[self.train_data['is_pop']==1]
            x_train, x_valid, y_train, y_valid = train_test_split(
                self.train_data_opt.drop(columns='item_quotation_price_num'),
                self.train_data_opt['item_quotation_price_num'],
                test_size=0.02,
                random_state=1)
            x_train = pd.concat([x_train, self.train_data_pop.drop(columns='item_quotation_price_num')])
            y_train = y_train.append(self.train_data_pop['item_quotation_price_num'])
            sample_weight = np.array(x_train['weight'])
            x_train, x_train_b, y_train = self.get_model_available_data(x_train, y_train, is_train=True, is_standard=is_standard)
            x_valid, x_valid_b, y_valid = self.get_model_available_data(x_valid, y_valid, is_standard=is_standard)
        else:
            x_train = self.train_data.drop(columns='item_quotation_price_num')
            y_train = self.train_data['item_quotation_price_num']

            sample_weight = np.array(x_train['weight'])

            x_train, x_train_b, y_train = self.get_model_available_data(x_train, y_train, is_train=True, is_standard=is_standard)
            x_valid, x_valid_b, y_valid = None, None, None

        x_recent_data, x_recent_data_b, y_recent_data = self.get_model_available_data(self.recent_data, self.recent_data['item_quotation_price_num'], is_standard=is_standard)

        return x_recent_data, x_recent_data_b, y_recent_data, sample_weight

    def train_ml_models(self):
        pass

    def train_linear_rgr(self):
        pass

    def train(self):

        # 数据检测
        os.system("echo 1 > /proc/sys/vm/drop_caches")
        starttime = time.time()
        if self.model_data is None:
            logger.critical('model_data is None!')
            return

        # 数据预处理
        self.prepare_data()
        # 构建模型训练特征
        self.build_features()

        # 训练经典机器学习模型
        self.train_ml_models()
        # 训练线性回归模型
        self.train_linear_rgr()

    def run(self):

        self.load_model_data()
        self.eliminate_anomaly()
        self.prepare_data()
        self.build_features()

        pass
